# 缺失值处理

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error
from sklearn.ensemble import RandomForestRegressor

data = pd.read_csv('../input/melbourne-housing-snapshot/melb_data.csv')

y = data.Price

melb_predictors = data.drop(['Price'], axis=1)

X = melb_predictors.select_dtypes(exclude=['object'])

X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.8, test_size=0.2, random_state=0)


def score_dataset(X_train, X_valid, y_train, y_valid):
    model = RandomForestRegressor(n_estimators=10, random_state=0)
    model.fit(X_train, y_train)
    preds = model.predict(X_valid)
    return mean_absolute_error(y_valid, preds)


# 移除法
def remove():
    # 获取缺失值的列
    cols_with_missing = [col for col in X_train.columns if X_train[col].isnull().any()]

    reduced_X_train = X_train.drop(cols_with_missing, axis=1)
    reduced_X_valid = X_valid.drop(cols_with_missing, axis=1)

    # 移除缺失值的平均绝对误差
    print(score_dataset(reduced_X_train, reduced_X_valid, y_train, y_valid))


# 平均值法，误差比移除要小
def simple_imputer():
    from sklearn.impute import SimpleImputer
    my_imputer = SimpleImputer()
    imputed_X_train = pd.DataFrame(my_imputer.fit_transform(X_train))
    imputed_X_valid = pd.DataFrame(my_imputer.transform(X_valid))

    imputed_X_train.columns = X_train.columns
    imputed_X_valid.columns = X_valid.columns

    print(score_dataset(imputed_X_train, imputed_X_valid, y_train, y_valid))


# remove()
simple_imputer()  # 178166.46269899711


# 方法3
def approach3():
    from sklearn.impute import SimpleImputer

    cols_with_missing = [col for col in X_train.columns if X_train[col].isnull().any()]

    X_train_plus = X_train.copy()
    X_valid_plus = X_valid.copy()

    for col in cols_with_missing:
        X_train_plus[col + '_was_missing'] = X_train_plus[col].isnull()
        X_valid_plus[col + '_was_missing'] = X_valid_plus[col].isnull()

    my_imputer = SimpleImputer()
    imputed_X_train_plus = pd.DataFrame(my_imputer.fit_transform(X_train_plus))
    imputed_X_valid_plus = pd.DataFrame(my_imputer.transform(X_valid_plus))

    imputed_X_train_plus.columns = X_train_plus.columns
    imputed_X_valid_plus.columns = X_valid_plus.columns

    print(score_dataset(imputed_X_train_plus, imputed_X_valid_plus, y_train, y_valid))

approach3()